package com.example.demo.spider;

import cn.hutool.core.util.ReUtil;
import com.alibaba.fastjson.JSON;
import com.example.demo.entity.News;
import com.example.demo.spider.pipeline.DbPipeline;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.Selectable;

import java.time.LocalDate;
import java.util.List;
import java.util.stream.Collectors;

@Component
public class News81Processor implements PageProcessor {
    private static final String BASE_URI = "http://81.cn/yw/";
    private static final String DETAIL = ".+content_\\d+.htm";
    private static final Site SITE = Site.me().setRetryTimes(3).setSleepTime(100);

    @Override
    public void process(Page page) {
        if (page.getUrl().regex(DETAIL).match()) {
            News news = resolve(page);
            page.putField("news", news);
        } else {
            List<Selectable> nodes = page.getHtml().xpath("//*[@id=\"main-news-list\"]/li").nodes();
            List<News> list = nodes.stream().map(this::resolve).filter(item -> {
                boolean flag = LocalDate.now().plusDays(-1).toString().equals(item.getDate().substring(0, 10));
                if (flag) page.addTargetRequest(item.getUrl());
                return flag;
            }).collect(Collectors.toList());
            page.putField("list", list);
        }
    }

    @Override
    public Site getSite() {
        return SITE;
    }

    private News resolve(Page page) {
        Html html = page.getHtml();
        String content = html.xpath("//div[@id=\"article-content\"]").get();
        List<String> images = html.xpath("//div[@id=\"article-content\"]//img/@src").all();
        images = images.stream().map(s -> BASE_URI + s.replaceAll("\\.\\./", "")).collect(Collectors.toList());
        List<String> video = html.xpath("//script[@title]").all();
        video = video.stream().map(s -> ReUtil.get("(http://.+mp[34])", s, 1)).collect(Collectors.toList());
        return new News().setContent(content)
                .setVideo(video.size() == 0 ? null : JSON.toJSONString(video))
                .setImages(images).setUrl(page.getUrl().get());
    }

    private News resolve(Selectable selectable) {
        String title = selectable.xpath("//div[@class=\"title\"]/text()").get();
        String date = selectable.xpath("//small/text()").get();
        String img = BASE_URI + selectable.xpath("//img/@src").get();
        String url = BASE_URI + selectable.xpath("//a/@href").get();
        return new News().setTitle(title).setImg(img).setDate(date).setUrl(url);
    }

    public static void main(String[] args) {
        Spider.create(new News81Processor()).addUrl("http://81.cn/yw").run();
    }

    @Autowired
    private DbPipeline dbPipeline;

    @Async
    public void start() {
        String[] urls = {
                "http://81.cn/yw", "http://81.cn/yw/index_2.htm"
        };
        Spider.create(this).addUrl(urls).addPipeline(dbPipeline).thread(5).run();
    }
}
